#-*- codeing = utf-8 -*-
#@Time : 2020/9/8 14:33
#@Author : 阳某
#@File : 经济学院.py
#@Software : PyCharm
import requests
import re
from openpyxl import Workbook
import json
import ast
from openpyxl import Workbook
from lxml import etree
wb = Workbook()
wa = wb.active
url = 'http://www.pbcsf.tsinghua.edu.cn/portal/list/index/id/10.html'
headers = {
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
}
wb = Workbook()
wa = wb.active
response = requests.get(url,headers = headers)
print(response.status_code)
# print(response.text)
data = re.findall('var data = \[(.*?)\];',response.text,re.S)
# print(data)
print(type(data))
print(len(data))
for da in data:
    print(da.strip())
    # da = json.loads(da)
    # da = eval(da)         #可以用来做数据类型的转换，也会做计算
    # da = ast.literal_eval(da)         #转换dict的时候，出于安全考虑对字符串进行类型转换的时候，最好使用ast.literal_eval()函数!
    print(da)
    # print(type(da))
    # print('+++'*30)
    urll = re.findall('"url":"(.*?)",',da,re.S)
    print(urll)
    for i in urll:
        line = []
        new_URL = 'http://www.pbcsf.tsinghua.edu.cn'+i
        print(new_URL)
        try:
            res = requests.get(new_URL, headers=headers, verify=False, timeout=3)
            print(res.status_code)
        except:
            print("超时了")
            continue
        res.encoding = res.apparent_encoding
        emai = re.findall(
            r"[\w!#$%&'*+/=?^_`{|}~-]+(?:\.[\w!#$%&'*+/=?^_`{|}~-]+)*@(?:[\w](?:[\w-]*[\w])?\.)+[\w](?:[\w-]*[\w])?",
            res.text, re.S)
        print(emai)
        e = etree.HTML(res.text)
        if len(emai) >= 2:
            emai = re.findall(
                r"[\w!#$%&'*+/=?^_`{|}~-]+(?:\.[\w!#$%&'*+/=?^_`{|}~-]+)*@(?:[\w](?:[\w-]*[\w])?\.)+[\w](?:[\w-]*[\w])?",
                res.text, re.S)[0]
            try:
                name = e.xpath('//h3/text()')[0].strip()  # ....4
                line.append(name)
                line.append(emai)
                print(line)
                wa.append(line)
            except:
                print("错了")
        else:
            print('没有邮箱')
wb.save('./金融学院')